require(RColorBrewer)
color <- brewer.pal(9, "Set1")

# Input files are expected in the working directory.
main.data <- read.csv("3-a_party_estimates.csv")
MPD.data <- read.csv("MPDataset_MPDS2024a.csv")
Kato.data <- read.csv("3-b_Kato_scores.csv")
UTAES.data <- read.csv("3-c_UTAES-based_scores.csv")

# merge manifesto project measures
MARPOR <- rep(NA, nrow(main.data))
for (i in 1: nrow(main.data)){
  # use Kato.data as a party-year crosswalk (see README for details)
  tmp <- subset(Kato.data, year_ws == main.data[i, "year"] & party_ws == main.data[i, "party"])
  if (nrow(tmp) == 0) {
    next
  } else if (is.na(tmp$party_mpd) == TRUE) {
    next
  } else {
    mpd.tmp <- subset(MPD.data, date == tmp$date_mpd & partyname == tmp$party_mpd)
    MARPOR[i] <- (mpd.tmp$per411 + mpd.tmp$per402 + mpd.tmp$per410 + mpd.tmp$per104 + 
                    mpd.tmp$per303 + mpd.tmp$per404 + mpd.tmp$per603 + mpd.tmp$per401) - 
      (mpd.tmp$per105 + mpd.tmp$per305 + mpd.tmp$per106 + mpd.tmp$per102 + 
         mpd.tmp$per202 + mpd.tmp$per602 + mpd.tmp$per203 + mpd.tmp$per103 + 
         mpd.tmp$per304)
  }
}
main.data <- cbind(main.data, MARPOR)

# merge Kato’s expert survey measure
Kato_point <- rep(NA, nrow(main.data))
for (i in 1: nrow(main.data)){
  tmp <- subset(Kato.data, year_ws == main.data[i, "year"] & party_ws == main.data[i, "party"])
  if (nrow(tmp) == 0) {
    next
  } else {
    Kato_point[i] <- tmp$kato_point
  }
}
main.data <- cbind(main.data, Kato_point)

# merge the UTAES measure
UTAS_ideology.mean <- rep(NA, nrow(main.data))
for (i in 1: nrow(main.data)){
  tmp <- subset(UTAES.data, year == main.data[i, "year"] - 1 & party == main.data[i, "party"])
  if (nrow(tmp) == 0) {
    next
  } else {
    UTAS_ideology.mean[i] <- tmp$ideology.mean
  }
}
main.data <- cbind(main.data, UTAS_ideology.mean)

p.color <- ifelse(main.data$party == "LDP_HOR", color[1],
                  ifelse(main.data$party == "JSP_HOR", color[2],
                         ifelse(main.data$party == "JCP_HOR", color[4], 
                                ifelse(main.data$party == "DPJ_HOR", color[3], 
                                       ifelse(main.data$party == "Komeito_HOR", color[5], 
                                              ifelse(main.data$party == "DSP_HOR", color[7], 
                                                     "gray30"))))))
p.pch <- ifelse(main.data$party == "LDP_HOR", "L",
                ifelse(main.data$party == "JSP_HOR", "S",
                       ifelse(main.data$party == "JCP_HOR", "C", 
                              ifelse(main.data$party == "DPJ_HOR", "d", 
                                     ifelse(main.data$party == "Komeito_HOR", "K", 
                                            ifelse(main.data$party == "DSP_HOR", "D", 
                                                   "o"))))))

# validation against MARPOR
table(is.na(main.data$MARPOR))  # number of missing vs non-missing observations
z.MARPOR <- lsfit(main.data$point, main.data$MARPOR, intercept = TRUE)  # fitted regression line
cor.estimate.MARPOR <- round(cor.test(main.data$point, main.data$MARPOR)$estimate, 2)  # correlation coefficient
cor.estimate.MARPOR <- format(cor.estimate.MARPOR, nsmall = 2)

main.data.MARPOR <- subset(main.data, ! is.na(MARPOR))
main.data.MARPOR$point.centered <- main.data.MARPOR$point - ave(main.data.MARPOR$point, main.data.MARPOR$party, FUN = mean)
main.data.MARPOR$MARPOR.centered <- main.data.MARPOR$MARPOR - ave(main.data.MARPOR$MARPOR, main.data.MARPOR$party, FUN = mean)
main.data.MARPOR <- subset(main.data.MARPOR, point.centered != 0)

# within-party (demeaned) correlation to focus on temporal movement within parties
z.MARPOR.centered <- lsfit(main.data.MARPOR$point.centered, main.data.MARPOR$MARPOR.centered, 
                           intercept = TRUE)
cor.estimate.MARPOR.centered <- round(cor.test(main.data.MARPOR$point.centered, 
                                               main.data.MARPOR$MARPOR.centered)$estimate, 2)
cor.estimate.MARPOR.centered <- format(cor.estimate.MARPOR.centered, nsmall = 2)

# validation against Kato's expert survey
table(is.na(main.data$Kato_point))  # number of missing vs non-missing observations
z.Kato <- lsfit(main.data$point, main.data$Kato_point, intercept = TRUE)  # fitted regression line
cor.estimate.Kato <- round(cor.test(main.data$point, main.data$Kato_point)$estimate, 2)  # correlation coefficient
cor.estimate.Kato <- format(cor.estimate.Kato, nsmall = 2)

main.data.Kato <- subset(main.data, ! is.na(Kato_point))
main.data.Kato$point.centered <- main.data.Kato$point - ave(main.data.Kato$point, main.data.Kato$party, FUN = mean)
main.data.Kato$Kato_point.centered <- main.data.Kato$Kato_point - ave(main.data.Kato$Kato_point, main.data.Kato$party, FUN = mean)
main.data.Kato <- subset(main.data.Kato, point.centered != 0)

z.Kato.centered <- lsfit(main.data.Kato$point.centered, main.data.Kato$Kato_point.centered, 
                         intercept = TRUE)
cor.estimate.Kato.centered <- round(cor.test(main.data.Kato$point.centered, 
                                             main.data.Kato$Kato_point.centered)$estimate, 2)
cor.estimate.Kato.centered <- format(cor.estimate.Kato.centered, nsmall = 2)

# validation against UTAES
table(is.na(main.data$UTAS_ideology.mean))  # number of missing vs non-missing observations
z.UTAS <- lsfit(main.data$point, main.data$UTAS_ideology.mean, intercept = TRUE)  # fitted regression line
cor.estimate.UTAS <- round(cor.test(main.data$point, main.data$UTAS_ideology.mean)$estimate, 2)  # correlation coefficient
cor.estimate.UTAS <- format(cor.estimate.UTAS, nsmall = 2)

main.data.UTAS <- subset(main.data, ! is.na(UTAS_ideology.mean))
main.data.UTAS$point.centered <- main.data.UTAS$point - ave(main.data.UTAS$point, main.data.UTAS$party, FUN = mean)
main.data.UTAS$UTAS_ideology.mean.centered <- main.data.UTAS$UTAS_ideology.mean - ave(main.data.UTAS$UTAS_ideology.mean, main.data.UTAS$party, FUN = mean)
main.data.UTAS <- subset(main.data.UTAS, point.centered != 0)

z.UTAS.centered <- lsfit(main.data.UTAS$point.centered, main.data.UTAS$UTAS_ideology.mean.centered, 
                         intercept = TRUE)
cor.estimate.UTAS.centered <- round(cor.test(main.data.UTAS$point.centered, 
                                             main.data.UTAS$UTAS_ideology.mean.centered)$estimate, 2)
cor.estimate.UTAS.centered <- format(cor.estimate.UTAS.centered, nsmall = 2)

# Figure 1: correlations with external measures (levels and within-party changes)
png("Figure_1.png", 
    width = 6, height = 4, units = "in", pointsize = 9, res = 1200)
par(mar = c(3.5, 3.5, 2, 0.5), pty = "s", lwd = 0.5)
layout(matrix(c(1:4, rep(5, 4), 6:9), 3, 4, byrow = TRUE), 
       widths = c(1, 1, 1, 0.3), heights = c(1, 0.1, 1))
plot(main.data$point, main.data$MARPOR, col = p.color, pch = p.pch, 
     main = "MARPOR", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.MARPOR, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.MARPOR), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("MARPOR ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
x_npc <- par("plt")[1] - 0.47
x_usr <- grconvertX(x_npc, from = "npc", to = "user")
mtext("(a)", line = 0.6, at = x_usr, adj = 0, cex = 1.1, font = 2)
plot(main.data$point, main.data$Kato_point, col = p.color, pch = p.pch, 
     main = "Kato\u2019s expert survey", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.Kato, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.Kato), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("Kato\u2019s expert survey measure", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
plot(main.data$point, main.data$UTAS_ideology.mean, col = p.color, pch = p.pch, 
     main = "UTAES", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.UTAS, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.UTAS), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("UTAES-based ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
par(mar = c(0, 0, 0, 0), pty = "m")
plot.new()
legend("center", 
       legend = c("LDP", "JSP/SDP", "JCP", "DSP", "Komeito", "DPJ", "others"),
       col = c(color[c(1, 2, 4, 8, 5, 3)], "gray30"),
       pch = c("L", "S", "C", "D", "K", "d", "o"), bty = "n")
plot.new()
par(mar = c(3.5, 3.5, 2, 0.5), pty = "s", lwd = 0.5)
plot(main.data.MARPOR$point.centered, main.data.MARPOR$MARPOR.centered, 
     col = ifelse(main.data.MARPOR$party == "LDP_HOR", color[1],
                  ifelse(main.data.MARPOR$party == "JSP_HOR", color[2],
                         ifelse(main.data.MARPOR$party == "JCP_HOR", color[4], 
                                ifelse(main.data.MARPOR$party == "DPJ_HOR", color[3], 
                                       ifelse(main.data.MARPOR$party == "Komeito_HOR", color[5], 
                                              ifelse(main.data.MARPOR$party == "DSP_HOR", color[8], 
                                                     "gray30")))))), 
     pch = ifelse(main.data.MARPOR$party == "LDP_HOR", "L",
                  ifelse(main.data.MARPOR$party == "JSP_HOR", "S",
                         ifelse(main.data.MARPOR$party == "JCP_HOR", "C", 
                                ifelse(main.data.MARPOR$party == "DPJ_HOR", "d", 
                                       ifelse(main.data.MARPOR$party == "Komeito_HOR", "K", 
                                              ifelse(main.data.MARPOR$party == "DSP_HOR", "D", 
                                                     "o")))))), 
     main = "MARPOR", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.MARPOR.centered, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.MARPOR.centered), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("MARPOR ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
x_npc <- par("plt")[1] - 0.47
x_usr <- grconvertX(x_npc, from = "npc", to = "user")
mtext("(b)", line = 0.6, at = x_usr, adj = 0, cex = 1.1, font = 2)
plot(main.data.Kato$point.centered, main.data.Kato$Kato_point.centered, 
     col = ifelse(main.data.Kato$party == "LDP_HOR", color[1],
                  ifelse(main.data.Kato$party == "JSP_HOR", color[2],
                         ifelse(main.data.Kato$party == "JCP_HOR", color[4], 
                                ifelse(main.data.Kato$party == "DPJ_HOR", color[3], 
                                       ifelse(main.data.Kato$party == "Komeito_HOR", color[5], 
                                              ifelse(main.data.Kato$party == "DSP_HOR", color[8], 
                                                     "gray30")))))), 
     pch = ifelse(main.data.Kato$party == "LDP_HOR", "L",
                  ifelse(main.data.Kato$party == "JSP_HOR", "S",
                         ifelse(main.data.Kato$party == "JCP_HOR", "C", 
                                ifelse(main.data.Kato$party == "DPJ_HOR", "d", 
                                       ifelse(main.data.Kato$party == "Komeito_HOR", "K", 
                                              ifelse(main.data.Kato$party == "DSP_HOR", "D", 
                                                     "o")))))), 
     main = "Kato\u2019s expert survey", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.Kato.centered, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.Kato.centered), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("Kato\u2019s expert survey measure", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
plot(main.data.UTAS$point.centered, main.data.UTAS$UTAS_ideology.mean.centered, 
     col = ifelse(main.data.UTAS$party == "LDP_HOR", color[1],
                  ifelse(main.data.UTAS$party == "JSP_HOR", color[2],
                         ifelse(main.data.UTAS$party == "JCP_HOR", color[4], 
                                ifelse(main.data.UTAS$party == "DPJ_HOR", color[3], 
                                       ifelse(main.data.UTAS$party == "Komeito_HOR", color[5], 
                                              ifelse(main.data.UTAS$party == "DSP_HOR", color[8], 
                                                     "gray30")))))), 
     pch = ifelse(main.data.UTAS$party == "LDP_HOR", "L",
                  ifelse(main.data.UTAS$party == "JSP_HOR", "S",
                         ifelse(main.data.UTAS$party == "JCP_HOR", "C", 
                                ifelse(main.data.UTAS$party == "DPJ_HOR", "d", 
                                       ifelse(main.data.UTAS$party == "Komeito_HOR", "K", 
                                              ifelse(main.data.UTAS$party == "DSP_HOR", "D", 
                                                     "o")))))), 
     main = "UTAES", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
abline(z.UTAS.centered, col = "gray", lty = "dashed")
legend("topleft", legend = paste0("r = ", cor.estimate.UTAS.centered), 
       bty = "n", x.intersp = 0)
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("UTAES-based ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
par(mar = c(0, 0, 0, 0), pty = "m")
plot.new()
legend("center", 
       legend = c("LDP", "JSP/SDP", "JCP", "DSP", "Komeito", "DPJ", "others"),
       col = c(color[c(1, 2, 4, 8, 5, 3)], "gray30"),
       pch = c("L", "S", "C", "D", "K", "d", "o"), bty = "n")
dev.off()

# JSP/SDP: within-party movement across measures
main.data.MARPOR.JSP <- subset(main.data.MARPOR, party == "JSP_HOR")
main.data.Kato.JSP <- subset(main.data.Kato, party == "JSP_HOR")
main.data.UTAS.JSP <- subset(main.data.UTAS, party == "JSP_HOR")

png("Figure_A1.png", 
    width = 5.4, height = 1.8, units = "in", pointsize = 9, res = 1200)
par(mar = c(3.5, 3.5, 2, 0.5), pty = "s", lwd = 0.5)
layout(matrix(1:3, 1, 3))
plot(main.data.MARPOR.JSP$point, main.data.MARPOR.JSP$MARPOR, type = "b", pch = "", 
     main = "MARPOR", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
text(main.data.MARPOR.JSP$point, main.data.MARPOR.JSP$MARPOR, substr(main.data.MARPOR.JSP$year - 1, 3, 4))
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("MARPOR ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
plot(main.data.Kato.JSP$point, main.data.Kato.JSP$Kato_point, type = "b", pch = "", 
     main = "Kato", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
text(main.data.Kato.JSP$point, main.data.Kato.JSP$Kato_point, substr(main.data.Kato.JSP$year - 1, 3, 4))
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("Kato\u2019s expert survey measure", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
plot(main.data.UTAS.JSP$point, main.data.UTAS.JSP$UTAS_ideology.mean, type = "b", pch = "", 
     main = "UTAS", xlab = "", ylab = "", 
     xaxt = "n", yaxt = "n")
text(main.data.UTAS.JSP$point, main.data.UTAS.JSP$UTAS_ideology.mean, substr(main.data.UTAS.JSP$year - 1, 3, 4))
mtext("This study\u2019s latent trait estimate", 1, 2.5, cex = 0.8)
mtext("UTAES-based ideology score", 2, 2.5, cex = 0.8)
axis(1, lwd = 0.5)
axis(2, lwd = 0.5)
dev.off()
